# -*- coding: utf-8 -*-
import requests
import json
from requests.auth import HTTPBasicAuth
import ast
import csv
import datetime
import time

def getMapping(host, port, username, password, index, doc_type):
    url = "http://" + host + ":" + port
    curr_url = url + "/" + index + "/" + doc_type + "/_mapping"
    headers = {'content-type': 'application/json',
               'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}
    requestsJson = requests.get(curr_url, headers=headers, auth=HTTPBasicAuth(username, password))
    return requestsJson.text


def getMapMapping(mappings, index, doc_type):
    dictMap = {};
    # print(mappings)
    data = json.loads(mappings)
    # print(data)
    mapping = data[index]['mappings'][doc_type]
    properties = data[index]['mappings'][doc_type]['properties']
    # print(mapping)
    keys = properties.keys()
    for key in keys:
        if "type" in properties[key]:
            dict2 = {key: properties[key]["type"]}
            dictMap.update(dict2)
        else:
            print("异常 type", )
    return dictMap


def get_all_es_data(host,port,index,doc_type,username, password,include,exclude):
    mappings = getMapping(host=host,port=port,index=index,doc_type=doc_type,username=username,password=password)
    mapData = getMapMapping(mappings=mappings,index=index,doc_type=doc_type)
    if len(include) > 0 :
        excludeMap =[]
        for exclude_key in mapData.keys():
            if exclude_key in include :
            else :
                excludeMap.append(exclude_key)
        if len(excludeMap) > 0 :
            for exclude_key in excludeMap:
                mapData.pop(exclude_key)
    if len(exclude) > 0 :
        for exclude_key in exclude :
            mapData.pop(exclude_key)

    # 过滤的query
    query = {
          "query": {
            "bool": {
              "must": [
                {

                }
              ]
            }
          },
          "_source": {
            "include": include,
            "exclude": exclude
          }
        }
    # 每次取的数据量
    size = 10000
    scroll_id = None
    count = 0
    finish = False
    startdateTime= datetime.datetime.now()
    print("开始时间 %s", startdateTime)
    try:
        file_name = str(startdateTime)+".csv"
        with open(file_name, 'w', newline='', encoding='utf-8') as file:
            csvwriter = csv.writer(file, delimiter=',',quotechar='\"', quoting=csv.QUOTE_ALL)
            csvwriter.writerow(mapData.keys())
            while True:
                if not scroll_id:
                    query["size"] = size
                    curr_url = "http://"+host+ ":"+port+"/"+index+"/"+doc_type+ '/_search?scroll=8m'
                    response = requests.post(curr_url, json.dumps(query), headers={'content-type': 'application/json'},auth=HTTPBasicAuth(username, password))
                else:
                    curr_url = "http://"+host+ ":"+port + '/_search/scroll?scroll=8m&scroll_id=' + scroll_id
                    response = requests.post(curr_url,auth=HTTPBasicAuth(username, password))
                if response:
                    response = json.loads(response.text)
                    scroll_id = response['_scroll_id']
                    total = response['hits']['total']
                    hits = response['hits']['hits']
                    length = len(response['hits']['hits'])
                    if not length or finish:
                        print("结束")
                        break
                    list = []  ## 空列表
                    for item in hits:
                        _source = item["_source"]
                        data = []
                        #list.append(_source['customer_id'])
                        for key in mapData.keys():
                            if key in _source.keys():
                                data.append(str(_source.get(key)))
                            else :
                                data.append("")
                        list.append(data)
                    csvwriter.writerows(list)
                    print(str(datetime.datetime.now())+"  list :",len(list),total)
            file.close()
    except Exception as err:
        print(err)
        pass

if __name__ == '__main__':
    host = "192.168.1.52"
    port = "9200"
    password = "changeme"
    username = "elastic"
    index = "myIndex"
    doc_type = "doc"
    include =["email"]        # 设置导出列
    exclude =[]                              # 设置不导出列 默认空集合
    get_all_es_data(host=host,port=port,index=index,doc_type=doc_type,username=username,password=password,include=include,exclude=exclude)

